﻿using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Text.RegularExpressions;
using PorterStemmerAlgorithm;

namespace Informaiton_Retrieval.com.ir.utli
{
    public class Tokenization
    {
        public static string[] FiltNumber(string[] input)
        {
            List<string> strList = new List<string>();
            foreach (string s in input)
            {

                if (!IsNumber(s))
                    strList.Add(s);
            }
            return strList.ToArray();
        }

        public static string[] FiltSame(string[] input)
        {
            List<string> strList = new List<string>();
            foreach (string s in input)
                if (!strList.Contains(s))
                    strList.Add(s);
            return strList.ToArray();
        }

        public static string[] LowerCase(string[] input)
        {
            List<string> strList = new List<string>();
            foreach (string s in input)
            {
                string ls = s.ToLower();

                strList.Add(ls);
            }
            return strList.ToArray();
        }

        public static string[] StopWord(string[] input)
        {
            List<string> strList = new List<string>();
            foreach (string s in input)
            {
                string ls = s;
                ls = Regex.Replace(s, stopwords, String.Empty);

                strList.Add(ls);
            }
            return strList.ToArray();
        }

        public static string[] DelEmpty(string[] input)
        {
            List<string> strList = new List<string>();
            foreach (string s in input)
            {
                if (!String.IsNullOrEmpty(s))
                    strList.Add(s);
            }
            return strList.ToArray();
        }

        public static string[] SortArray(string[] input)
        {
            List<string> strList = new List<string>();
            foreach (string s in input)
                strList.Add(s);
            strList.Sort();
            return strList.ToArray();
        }

        public static string[] Stemming(string[] input)
        {
            StemmerInterface sf = new PorterStemmer();
            List<string> strList = new List<string>();
            foreach (string s in input)
            {
                string ss = sf.stemTerm(s);
                strList.Add(ss);
            }
            return strList.ToArray();
        }

        static bool IsNumber(string text)
        {
            Regex regex = new Regex(@"^[-+]?[0-9]*\.?[0-9]+$");
            return regex.IsMatch(text);
        }

        static readonly string stopwords = @"\b(?:a|an|the|and|but|if|or|because|as|until|while|of|at|by|for|with|about|against|between|into|through|
                                            during|before|after|above|below|to|from|up|down|in|out|on|off|over|under|again|further|then|once|
                                            here|there|when|where|why|how|all|any|both|each|few|more|most|other|some|such|no|nor|not|only|
                                            so|than|too|very|one|every|least|less|many|now|ever|never|also|just|whether|like|well|back|even|
                                            still|way|since|another|however|let's|that's|who's|what's|here's|there's|when's|where's|
                                            why's|how's|won't|wouldn't|shan't|shouldn't|can't|cannot|couldn't|mustn't|isn't|aren't|wasn't|
                                            weren't|hasn't|haven't|hadn't|doesn't|don't|didn't|i'm|you're|he's|she's|it's|we're|they're|i've|
                                            you've|we've|they've|i'd|you'd|he'd|she'd|we'd|they'd|i'll|you'll|he'll|she'll|we'll|they'll|me|my|
                                            myself|we|us|our|ours|ourself|you|your|yours|yourself|yourselves|he|him|his|hiself|she|her|hers|
                                            herself|it|its|itself|they|them|their|theirs|themselves|what|which|who|whom|this|that|these|those|
                                            am|is|are|was|were|be|been|being|have|has|had|having|do|does|did|doing|will|would|should|could|ought|
                                            may|might|must|shall)\b";

    }
}
